﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data;
using SEO.Business;
using System.Windows.Forms;
using SEO.Entity;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading;

namespace WebCrawler
{
    /// <summary>
    /// 服务端爬虫主程序
    /// </summary>
    public class CrawlerMain
    {
        /// <summary>
        /// 抓取页数
        /// </summary>
        public static int nPage = 2;

        /// <summary>
        /// 配置信息
        /// </summary>
        public static SysConfig sysInfo = null;

        /// <summary>
        /// 当天的第几次
        /// </summary>
        public static int nSysRunIndex = 1;

        /// <summary>
        /// 运行记录ID
        /// </summary>
        public static int nSysRunLogID = 0;

        /// <summary>
        /// 运行开始时间
        /// </summary>
        public static DateTime dSysBeginTime = DateTime.Now;

        /// <summary>
        /// 百度url
        /// </summary>
        public static string strBaiduUrl = "http://www.baidu.com/s?wd={0}&pn={1}&ie=utf-8";

        /// <summary>
        /// 百度收入多少天才合要求
        /// </summary>
        public static int nBaiduDay = 2;
 
        [STAThread]
        public static void Main(string[] args)
        {

            //初始化配置信息
            Console.WriteLine("初始化配置信息");
            if (!InitConfig())
            {
                return;
            }
            // CreateUserArticleReport();
            //CreateKeywordsReport();
            //UpdateWebSEOType();
            //UpdateWebSEOUrl();
            //CreateUserArticleReport();
            //写入日志
            Console.WriteLine("创建日志");
            if (InserSysRunLog())
            {
                //关键字收录分析
                Console.WriteLine("关键字收录分析");
                CrawlerKeywords();
                //关键字收录表报生成
                Console.WriteLine("关键字收录报表生成");
                CreateKeywordsReport();
                //网编文章收录分析
                Console.WriteLine("网编文章收录分析");
                CrawlerArticle();
                if (nSysRunIndex == 1)
                {
                    //每天7点前锁定关键字
                    if (dSysBeginTime < Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd 7:00:00")))
                    {
                        Console.WriteLine("关键字锁定状态分析");
                        CrawlerKeywordsSysLock();
                    }
                    Console.WriteLine("用户文章报表生成");
                    CreateUserArticleReport();
                }
                if (CheckAccountDay())
                {
                    //网编文章提成分析
                    Console.WriteLine("网编文章提成分析");
                    CrawlerMemony();
                }
                //更新运行日志
                UpdateSysRunLog();
            }
            Console.WriteLine("完成");
           // Console.ReadLine();
        }

        #region 初始化配置信息
        /// <summary>
        /// 初始化配置信息
        /// </summary>
        public static bool InitConfig()
        {
            try
            {
                sysInfo = new BisSysConfig().GetSysConfig();
                if (sysInfo != null)
                {
                    nPage = sysInfo.BaiduPage.Value;
                    nBaiduDay = sysInfo.BaiduDay.Value;
                }
                else
                {
                    Console.WriteLine("初始化配置信息出错");
                    return false;
                }
                return true;
            }
            catch (Exception ex)
            {
                Console.WriteLine("初始化配置信息出错");
                Common.Log(ex.Message.ToString());
                return false;
            }
        }

        /// <summary>
        /// 创建日志
        /// </summary>
        /// <returns></returns>
        public static bool InserSysRunLog()
        {
            try
            {
                BisSysRunLog bisSysRunLog = new BisSysRunLog();
                //查找当天最后一条运行记录
                DateTime dBeginTime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd 00:00:00"));
                DateTime dEndTime = dBeginTime.AddDays(1);
                DataTable dtSysRunLog = bisSysRunLog.GetLastLog(dBeginTime, dEndTime);
                if (dtSysRunLog != null && dtSysRunLog.Rows.Count > 0)
                {
                    nSysRunIndex = Convert.ToInt32(dtSysRunLog.Rows[0]["RunIndex"]);
                    nSysRunIndex++;
                }
                else
                {
                    nSysRunIndex = 1;
                }
                dtSysRunLog.Dispose();
                //新增运行记录
                SysRunLog info = new SysRunLog();
                info.BeginTime = dSysBeginTime;
                info.RunIndex = nSysRunIndex;
                nSysRunLogID = bisSysRunLog.Insert(info);
                return nSysRunLogID > 0;
            }
            catch (Exception ex)
            {
                Console.WriteLine("创建日志出错");
                Common.Log(ex.Message.ToString());
                return false;
            }
        }
        /// <summary>
        /// 更新运行日志
        /// </summary>
        public static void UpdateSysRunLog()
        {
            SysRunLog info = new SysRunLog();
            info.EndTime = DateTime.Now;
            info.ID = nSysRunLogID;
            new BisSysRunLog().Update(info);
        }
        #endregion 初始化配置信息

        #region 关键字收录分析
        /// <summary>
        /// 关键字收录分析
        /// </summary>
        public static void CrawlerKeywords()
        {
            //医院
            DataTable dtMedical = new BisMedical().GetList();
            //关键字
            DataTable dtKeywords = new BisKeywords().GetAllList();
            int nMedicalID = 0;
            string strMedicalKey = string.Empty;
            int nKeywordsID = 0;
            string strKeywords = string.Empty;
            string strUrl = string.Empty;
            string strHtml = string.Empty;

            string strPageTitle = string.Empty;
            string strPageUrl = string.Empty;
            string strPageInfo = string.Empty;
            string strBaiduPageUrl = string.Empty;
            string strPageHtml = string.Empty;
            bool bIsSEO = false;
            int nRowIndex = 0;
            int nAllIndex = 0;
            DateTime dBeginTime = Convert.ToDateTime(dSysBeginTime.ToString("yyyy-MM-dd 00:00:00"));
            DateTime dEndTime = Convert.ToDateTime(dSysBeginTime.AddDays(1).ToString("yyyy-MM-dd 00:00:00"));

            BisWebSEO bisWebSEO = new BisWebSEO();
            BisNewsWeb bisNewsWeb = new BisNewsWeb();
            BisOurWeb bisOurWeb = new BisOurWeb();
            if (dtMedical != null && dtMedical.Rows.Count > 0)
            {
                int nCountKeyword = 0;
                foreach (DataRow drKeywords in dtKeywords.Rows)
                {
                    nCountKeyword++;
                    nKeywordsID = Convert.ToInt32(drKeywords["ID"]);
                    strKeywords = drKeywords["Name"].ToString();
                    Console.WriteLine("抓取关键字：第" + nCountKeyword.ToString() + "个 " + strKeywords);
                    nAllIndex = 0;
                    //按配置页数抓取
                    for (int i = 0; i < nPage; i++)
                    {
                        nRowIndex = 0;
                        //抓取页面信息
                        strUrl = string.Format(strBaiduUrl, Common.UrlEncode(strKeywords), i * 10);
                        strHtml = Common.GetPageHTML(strUrl);
                        if (!string.IsNullOrEmpty(strHtml))
                        {
                            //分析页面
                            BaseAction baseAction = new BaseAction();
                            baseAction.CreateHtmlDoc(strHtml);
                            for (int j = 1; j <= 10; j++)
                            {
                                try
                                {
                                    bIsSEO = false;
                                    strPageTitle = string.Empty;
                                    strPageInfo = string.Empty;
                                    strPageUrl = string.Empty;
                                    strBaiduPageUrl = string.Empty;
                                    strPageHtml = string.Empty;
                                    nAllIndex++;
                                    nRowIndex++;
                                    //分析标题部分
                                    HtmlElement eleItem = baseAction.htmldoc.GetElementById(((i * 10) + j).ToString());
                                    if (eleItem != null)
                                    {
                                        List<HtmlElement> elemTD = baseAction.GetHtmlByClass(eleItem, "td", "f");
                                        if (elemTD.Count > 0)
                                        {
                                            HtmlElementCollection rowList = elemTD[0].Children;
                                            bool bTitle = false;
                                            bool bInfo = false;
                                            foreach (HtmlElement row in rowList)
                                            {
                                                switch (row.TagName.ToLower())
                                                {
                                                    //标题
                                                    case "h3":
                                                        if (!bTitle)
                                                        {
                                                            HtmlElementCollection aLink = row.Children;
                                                            if (aLink.Count > 0)
                                                            {
                                                                strPageTitle = aLink[0].InnerText.Trim();
                                                                strBaiduPageUrl = aLink[0].GetAttribute("href").Trim();
                                                            }
                                                            bTitle = true;
                                                        }
                                                        break;
                                                    //内容
                                                    case "div":
                                                        if (!bInfo)
                                                        {
                                                            strPageInfo = row.InnerText.Trim();
                                                            bInfo = true;
                                                        }
                                                        break;
                                                }
                                                if (bTitle && bInfo)
                                                    break;
                                            }

                                            //获取文章ID
                                            int nWebSeoID = 0;
                                            //匹配医院
                                            foreach (DataRow drMedical in dtMedical.Rows)
                                            {
                                                if (nSysRunIndex > 1 && !Convert.ToBoolean(drMedical["Status"]))
                                                    continue;
                                                //已匹配到医院就不匹配
                                                if (bIsSEO)
                                                    break;
                                                bIsSEO = false;
                                                nMedicalID = Convert.ToInt32(drMedical["ID"]);
                                                if (strPageTitle.IndexOf("百度知道") > 0)
                                                    break;
                                                if (strPageTitle.IndexOf("搜搜问问") > 0)
                                                    break;
                                                //匹配特征
                                                string[] keyList = drMedical["Keys"].ToString().Split('|');
                                                foreach (string temKey in keyList)
                                                {
                                                    if (bIsSEO)
                                                        break;
                                                    strMedicalKey = temKey.Trim();
                                                    if (!string.IsNullOrEmpty(strMedicalKey))
                                                    {
                                                        //匹配百度标题
                                                        if (strPageTitle.IndexOf(strMedicalKey) >= 0)
                                                            bIsSEO = true;
                                                        //匹配百度介绍
                                                        if (!bIsSEO && strPageInfo.IndexOf(strMedicalKey) >= 0)
                                                            bIsSEO = true;
                                                        if (bIsSEO)
                                                        {
                                                            //根据百度路径获取真实网址
                                                            if (string.IsNullOrEmpty(strPageUrl))
                                                                strPageUrl = Common.GetFirstUrl(strBaiduPageUrl);
                                                            if (!string.IsNullOrEmpty(strPageUrl))
                                                            {
                                                                if (nWebSeoID == 0)
                                                                {
                                                                    //验证文章是否存在
                                                                    nWebSeoID = bisWebSEO.GetWebSEOID(strPageUrl);
                                                                    if (nWebSeoID == 0)
                                                                    {
                                                                        //不存在新增
                                                                        WebSEO Info = new WebSEO();
                                                                        Info.WebTitle = strPageTitle;
                                                                        Info.WebInfo = strPageInfo;
                                                                        Info.BaiduUrl = strBaiduPageUrl;
                                                                        Info.WebUrl = strPageUrl;
                                                                        Info.WebType = 2;
                                                                        Info.CreateTime = dSysBeginTime;
                                                                        if (Convert.ToBoolean(drMedical["Status"]))
                                                                        {
                                                                            //验证是否新闻源
                                                                            string strTopDomain = Common.GetTopDomain(strPageUrl);
                                                                            if (Common.CheckNewsWeb(strPageUrl) > 0)
                                                                            {
                                                                                Info.WebType = 1;
                                                                            }
                                                                            else
                                                                            {
                                                                                //验证是否站内
                                                                                if (bisOurWeb.CheckOurWeb(strTopDomain))
                                                                                {
                                                                                    Info.WebType = 3;
                                                                                }
                                                                            }
                                                                        }
                                                                        nWebSeoID = bisWebSEO.Insert(Info);
                                                                    }
                                                                }
                                                                if (nWebSeoID > 0)
                                                                {
                                                                    //记录关系
                                                                    DataTable dtKeysRelation = bisWebSEO.GetKeysRelation(nMedicalID, nKeywordsID, nWebSeoID, dBeginTime, dEndTime);
                                                                    int nKeysRelationID = 0;
                                                                    if (dtKeysRelation != null && dtKeysRelation.Rows.Count > 0)
                                                                    {
                                                                        nKeysRelationID = Convert.ToInt32(dtKeysRelation.Rows[0]["ID"]);
                                                                        if (Convert.ToInt32(dtKeysRelation.Rows[0]["FinalPageIndex"]) < (i + 1))
                                                                            bisWebSEO.UpdateKeysRelation(nKeysRelationID, Convert.ToInt32(dtKeysRelation.Rows[0]["FinalPageIndex"]));
                                                                    }
                                                                    else
                                                                    {
                                                                        nKeysRelationID = bisWebSEO.InsertKeysRelation(nMedicalID, nKeywordsID, nWebSeoID, (i + 1), nRowIndex, nAllIndex, nSysRunIndex, (i + 1), dSysBeginTime);
                                                                    }
                                                                    dtKeysRelation.Dispose();

                                                                    //记录当天收录情况
                                                                    if (nKeysRelationID > 0)
                                                                    {
                                                                        bisWebSEO.InsertKeysRelationDay(nKeysRelationID, (i + 1), nRowIndex, nAllIndex, nSysRunIndex, dSysBeginTime);
                                                                    }
                                                                }
                                                                Console.WriteLine("第" + (i + 1).ToString() + "页 第" + nRowIndex.ToString() + "位 收录");
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    }

                                }
                                catch (Exception ex)
                                {
                                    Common.Log("分析SEO关键字(" + strKeywords + ")失败" + ex.Message);
                                }
                            }
                            baseAction.DisDoc();
                        }
                    }
                }
            }
            dtMedical.Dispose();
            dtKeywords.Dispose();
        }
        #endregion

        #region 关键字收录报表生成
        /// <summary>
        /// 关键字收录报表生成
        /// </summary>
        public static void CreateKeywordsReport()
        {
            //dSysBeginTime = Convert.ToDateTime("2013-7-15 21:17:36");
            //nSysRunIndex = 2;
            BisKeywordsReport bisInfo = new BisKeywordsReport();
            DateTime dBeginDay = Convert.ToDateTime(dSysBeginTime.ToString("yyyy-MM-dd"));
            DateTime dEndDay = Convert.ToDateTime(dSysBeginTime.AddDays(1).ToString("yyyy-MM-dd"));
            //获取收录信息
            DataTable dt = bisInfo.GetList(nSysRunIndex, dBeginDay, dEndDay);
            DataTable dtMedical = new BisMedical().GetList();
            DataTable dtKeywords = new BisKeywords().GetAllList();
            //删除原有的
            bisInfo.Delete(nSysRunIndex, dBeginDay, dEndDay);
            foreach (DataRow drMedical in dtMedical.Rows)
            {
                if (nSysRunIndex > 1 && !Convert.ToBoolean(drMedical["Status"]))
                    break;
                foreach (DataRow drKeywords in dtKeywords.Rows)
                {
                    try
                    {
                        Console.WriteLine(drMedical["Name"].ToString() + " 关键字：" + drKeywords["Name"].ToString());
                        DataRow[] drList = dt.Select("MedicalID=" + drMedical["ID"].ToString() + " and KeywordsID=" + drKeywords["ID"].ToString());
                        KeywordsReport info = new KeywordsReport();
                        info.MedicalID = Convert.ToInt32(drMedical["ID"]);
                        info.KeywordsID = Convert.ToInt32(drKeywords["ID"]);
                        info.DayIndex = nSysRunIndex;
                        info.CreateTime = dSysBeginTime;
                        info.FirstCount = 0;
                        info.SecondCount = 0;
                        info.ThirdCount = 0;
                        info.NewsCount = 0;
                        info.LinkCount = 0;
                        foreach (DataRow dr in drList)
                        {
                            info.SEOInfo += (string.IsNullOrEmpty(info.SEOInfo) ? "" : "，") + dr["AllIndex"].ToString();
                            //统计页数
                            switch (Convert.ToInt32(dr["PageIndex"]))
                            {
                                case 1:
                                    info.FirstCount++;
                                    break;
                                case 2:
                                    info.SecondCount++;
                                    break;
                                case 3:
                                    info.ThirdCount++;
                                    break;
                            }
                            //统计类型
                            switch (Convert.ToInt32(dr["WebType"]))
                            {
                                case 1:
                                    info.NewsCount++;
                                    break;
                                case 2:
                                    info.LinkCount++;
                                    break;
                            }
                        }
                        bisInfo.Insert(info);
                    }
                    catch (Exception ex)
                    {
                        string strErrorInfo = "\r\n生成报表错误描述："
                           + "\r\n医院：" + drMedical["Name"].ToString()
                           + "\r\n关键字:" + drKeywords["Name"].ToString()
                           + "\r\n报表时间：" + dSysBeginTime.ToString()
                           + "\r\n当天次数：" + nSysRunIndex
                           + "\r\n出错信息：" + ex.Message;
                        Common.Log(strErrorInfo);
                    }

                }
            }

        }
        #endregion

        #region 关键字锁定状态分析
        /// <summary>
        /// 关键字锁定状态分析
        /// </summary>
        public static void CrawlerKeywordsSysLock()
        {
            try
            {
                BisKeywords bisKeywords = new BisKeywords();
                //将所有关键字系统锁定解除
                bisKeywords.UpdateAllSysLock(false);
                DateTime dBeginTime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd 00:00:00"));
                DataTable dtKeywords = bisKeywords.GetKeywordsByDay(dBeginTime, dBeginTime.AddDays(1));
                if (dtKeywords != null && dtKeywords.Rows.Count > 0)
                {
                    foreach (DataRow drKeywords in dtKeywords.Rows)
                    {
                        try
                        {
                            //首页关键字收录大于等于指定值则锁定
                            if (Convert.ToInt32(drKeywords["DayCount"]) >= sysInfo.LockNumber)
                            {
                                bisKeywords.UpdateSysLock(Convert.ToInt32(drKeywords["ID"]), true);
                            }
                        }
                        catch (Exception ex)
                        {
                            Common.Log("关键字锁定(" + drKeywords["ID"].ToString() + ")失败:" + ex.Message);
                        }
                    }
                }
                dtKeywords.Dispose();
            }
            catch (Exception ex)
            {
                Common.Log("关键字锁定状态分析失败:" + ex.Message);
            }
        }
        #endregion

        #region 用户文章报表生成
        /// <summary>
        /// 用户文章报表生成
        /// </summary>
        public static void CreateUserArticleReport()
        {
            //获取用户
            DateTime dBeginTime = Common.GetAccountDay(sysInfo);
            DateTime dNowTime = DateTime.Now;
            TimeSpan tSpan = dNowTime - dBeginTime;
            DataTable dtUser = new BisAdminInfo().GetAllUserList();
            BisArticleSEO bisArticleSEO = new BisArticleSEO();
            BisUserArticleReport bisInfo = new BisUserArticleReport();
            DateTime dEndTime = Common.GetAccountDay(sysInfo).AddMonths(1);
            int nDay = tSpan.Days;
            if (nDay > 0)
            {
                for (int i = 0; i < nDay; i++)
                {
                    DateTime dDayTime = dBeginTime.AddDays(i);
                    bisInfo.Delete(dDayTime);
                    foreach (DataRow drUser in dtUser.Rows)
                    {
                        Console.WriteLine("生成" + dDayTime.ToString() + " " + drUser["Name"].ToString() + "文章统计数据");
                        UserArticleReport Info = new UserArticleReport();
                        Info.AdminID = Convert.ToInt32(drUser["ID"]);
                        Info.OurWebCount = bisArticleSEO.CountArticle(Info.AdminID.Value, 3, -1, dDayTime, dDayTime.AddDays(1));
                        Info.OurWebUserCount = bisArticleSEO.CountArticle(Info.AdminID.Value, 3, 1, dDayTime, dDayTime.AddDays(1));
                        Info.NewsAllCount = bisArticleSEO.CountArticle(Info.AdminID.Value, 1, -1, dDayTime, dDayTime.AddDays(1));
                        Info.NewsCount1 = 0;
                        Info.NewsAllSEOCount = 0;
                        Info.NewsCount2 = 0;
                        Info.LinkAllCount = bisArticleSEO.CountArticle(Info.AdminID.Value, 2, -1, dDayTime, dDayTime.AddDays(1));
                        Info.LinkAllSEOCount = 0;
                        Info.LinkCount1 = 0;
                        Info.LinkCount2 = 0;
                        Info.CreateTime = dDayTime;
                        DataTable dtArticle = bisArticleSEO.GetUserArticleList(Info.AdminID.Value, true, dDayTime, dDayTime.AddDays(1));
                        foreach (DataRow drArticle in dtArticle.Rows)
                        {
                            int nUrlType = Convert.ToInt32(drArticle["UrlType"]);
                            int nRe = 0;
                            int nRe1 = 0;
                            //分析关键字1
                            if (drArticle["KeywordsID"] != DBNull.Value)
                                nRe = bisArticleSEO.CountArticleReport(Convert.ToInt32(drArticle["WebSEOID"]), Convert.ToInt32(drArticle["KeywordsID"]), dDayTime.AddDays(1), dEndTime, nBaiduDay);
                            if (nRe > 0)
                            {
                                switch (nUrlType)
                                {
                                    case 1:
                                        if (nRe == 1)
                                        {
                                            Info.NewsCount1 += 1;
                                        }
                                        else if (nRe == 2)
                                        {
                                            Info.NewsCount2 += 1;
                                        }
                                        break;
                                    case 2:
                                        if (nRe == 1)
                                        {
                                            Info.LinkCount1 += 1;
                                        }
                                        else if (nRe == 2)
                                        {
                                            Info.LinkCount2 += 1;
                                        }
                                        break;
                                }
                            }

                            //分析关键字2
                            if (drArticle["KeywordsID1"] != DBNull.Value)
                                nRe1 = bisArticleSEO.CountArticleReport(Convert.ToInt32(drArticle["WebSEOID"]), Convert.ToInt32(drArticle["KeywordsID1"]), dDayTime.AddDays(1), dEndTime, nBaiduDay);
                            if (nRe1 > 0)
                            {
                                switch (nUrlType)
                                {
                                    case 1:
                                        if (nRe1 == 1)
                                        {
                                            Info.NewsCount1 += 1;
                                        }
                                        else if (nRe1 == 2)
                                        {
                                            Info.NewsCount2 += 1;
                                        }
                                        break;
                                    case 2:
                                        if (nRe1 == 1)
                                        {
                                            Info.LinkCount1 += 1;
                                        }
                                        else if (nRe1 == 2)
                                        {
                                            Info.LinkCount2 += 1;
                                        }
                                        break;
                                }
                            }

                            if (nRe > 0 || nRe1 > 0)
                            {
                                switch (nUrlType)
                                {
                                    case 1:
                                        Info.NewsAllSEOCount += 1;
                                        break;
                                    case 2:
                                        Info.LinkAllSEOCount += 1;
                                        break;
                                }
                            }
                        }
                        dtArticle.Dispose();
                        bisInfo.Insert(Info);
                    }
                }
            }
            dtUser.Dispose();
        }
        #endregion

        #region 网编文章收录分析
        /// <summary>
        /// 网编文章收录分析
        /// </summary>
        public static void CrawlerArticle()
        {
            try
            {
                BisWebSEO bisWebSEO = new BisWebSEO();
                BisArticleSEO bisArticleSEO = new BisArticleSEO();
                //获取状态还是没收录过的文章
                DataTable dtArticleSeo = null;// new BisArticleSEO().GetCrawlerArticleList(false, false, Common.GetAccountDay(sysInfo));
                int nWebSEOID = 0;
                DateTime dBeginTime = DateTime.Now;
                int nKeywordsID = 0;
                int nKeywordsID1 = 0;
                foreach (DataRow drArticleSeo in dtArticleSeo.Rows)
                {
                    try
                    {
                        Console.WriteLine("检测文章网址：" + drArticleSeo["ArticleUrl"].ToString());
                        if (Convert.ToInt32(drArticleSeo["UrlType"]) == 3)
                        {
                            //站内类型则只验证是否收录
                            if (CheckBaiDu(drArticleSeo["ArticleUrl"].ToString()))
                                bisArticleSEO.UpdateStatus(Convert.ToInt32(drArticleSeo["ID"]), true);
                        }
                        else
                        {
                            //验证百度收录关系
                            dBeginTime = Convert.ToDateTime(Convert.ToDateTime(drArticleSeo["CreateTime"]).AddDays(1).ToString("yyyy-MM-dd"));
                            int.TryParse(drArticleSeo["KeywordsID"].ToString(), out nKeywordsID);
                            int.TryParse(drArticleSeo["KeywordsID1"].ToString(), out nKeywordsID1);
                            bisArticleSEO.CheckBaiduSEO(Convert.ToInt32(drArticleSeo["ID"]), Convert.ToInt32(drArticleSeo["SortID"]), nKeywordsID, nKeywordsID1, drArticleSeo["ArticleUrl"].ToString(), dBeginTime, ref nWebSEOID);
                        }
                    }
                    catch (Exception ex)
                    {
                        Common.Log("检测文章网址(" + drArticleSeo["ArticleUrl"].ToString() + ")失败:" + ex.Message);
                    }
                }
                dtArticleSeo.Dispose();
            }
            catch (Exception ex)
            {
                Common.Log("网编文章收录分析:" + ex.Message);
            }
        }
        #endregion

        #region 网编文章提成分析
        /// <summary>
        /// 网编文章提成分析
        /// </summary>
        public static void CrawlerMemony()
        {
            try
            {
                BisArticleSEO bisArticleSEO = new BisArticleSEO();
                DateTime dEndTime = Common.GetAccountDay(sysInfo);
                //获取已收录的文章
                DataTable dtArticleSeo = new BisArticleSEO().GetList(true, false, dEndTime.AddMonths(-1));
                int nKeywordsID = 0;
                int nKeywordsID1 = 0;
                string strTopDomain = string.Empty;
                foreach (DataRow drArticleSeo in dtArticleSeo.Rows)
                {
                    try
                    {
                        Console.WriteLine("检测文章：" + drArticleSeo["AritcleTitle"].ToString());
                        if (Convert.ToInt32(drArticleSeo["UrlType"]) == 3)
                        {
                            //站内类型只要有收录就提成
                            bisArticleSEO.UpdateIsPass(Convert.ToInt32(drArticleSeo["ID"]), true, sysInfo.OurWebMomey.Value);
                        }
                        else
                        {
                            int.TryParse(drArticleSeo["KeywordsID"].ToString(), out nKeywordsID);
                            int.TryParse(drArticleSeo["KeywordsID1"].ToString(), out nKeywordsID1);
                            strTopDomain = Common.GetTopDomain(drArticleSeo["ArticleUrl"].ToString());
                            //非站内类型提成计算
                            bisArticleSEO.CountArticleMomey(
                                Convert.ToInt32(drArticleSeo["ID"]),
                                Convert.ToInt32(drArticleSeo["UrlType"]),
                                Convert.ToInt32(drArticleSeo["WebSEOID"]),
                                strTopDomain,
                                nKeywordsID,
                                nKeywordsID1,
                                Convert.ToDateTime(Convert.ToDateTime(drArticleSeo["CreateTime"]).AddDays(1).ToString("yyyy-MM-dd 00:00:00")),
                                dEndTime,
                                sysInfo);
                        }
                    }
                    catch (Exception ex)
                    {
                        Common.Log("网编文章提成分析(" + drArticleSeo["AritcleTitle"].ToString() + ")失败:" + ex.Message);
                    }
                }
                dtArticleSeo.Dispose();
                //打上结算标记
                bisArticleSEO.UpdateIsAccount();
            }
            catch (Exception ex)
            {
                Common.Log("网编文章提成分析:" + ex.Message);
            }

        }
        #endregion

        /// <summary>
        /// 更新所有收录网址Url
        /// </summary>
        public static void UpdateWebSEOUrl()
        {
            BisWebSEO bisInfo = new BisWebSEO();
            BisNewsWeb bisNewsWeb = new BisNewsWeb();
            BisOurWeb bisOurWeb = new BisOurWeb();
            DataTable dt = bisInfo.GetAllList();
            int nCount = 0;
            foreach (DataRow dr in dt.Rows)
            {
                nCount++;
                Console.WriteLine(nCount + " " + dr["ID"].ToString());
                string strPageUrl = dr["WebUrl"].ToString();
                //去"/"
                if (strPageUrl.Substring(strPageUrl.Length - 1, 1) == "/")
                {
                    strPageUrl = strPageUrl.Substring(0, strPageUrl.Length - 1);
                    bool bResult = bisInfo.UpdateWebUrl(Convert.ToInt32(dr["ID"]), strPageUrl);
                    Console.WriteLine(" 存在" + (bResult ? "成功" : "失败"));
                }
                else
                {
                    Console.WriteLine("不存在");
                }


            }
        }
        /// <summary>
        /// 更新所有收录网址的类型
        /// </summary>
        public static void UpdateWebSEOType()
        {
            BisWebSEO bisInfo = new BisWebSEO();
            BisNewsWeb bisNewsWeb = new BisNewsWeb();
            BisOurWeb bisOurWeb = new BisOurWeb();
            DataTable dt = bisInfo.GetAllList();
            int nCount = 0;
            foreach (DataRow dr in dt.Rows)
            {
                nCount++;
                Console.WriteLine(nCount + " " + dr["ID"].ToString());
                int nWebType = 2;
                string strTopDomain = Common.GetTopDomain(dr["WebUrl"].ToString());
                if (bisNewsWeb.CheckNewsWeb(strTopDomain))
                {
                    nWebType = 1;
                }
                else
                {
                    //验证是否站内
                    if (bisOurWeb.CheckOurWeb(strTopDomain))
                    {
                        nWebType = 3;
                    }
                }
                bool bResult = bisInfo.UpdateWebType(Convert.ToInt32(dr["ID"]), nWebType);
                Console.WriteLine((nWebType == 1 ? "新闻源" : (nWebType == 2 ? "外链" : "站内")) + " " + (bResult ? "成功" : "失败"));
            }
        }

        /// <summary>
        /// 验证是否结算日
        /// </summary>
        /// <returns></returns>
        public static bool CheckAccountDay()
        {
            if (nSysRunIndex == 1)
            {
                string strAccountDay = Common.GetAccountDay(sysInfo).ToString("yyyy-MM-dd");
                string strNowDay = DateTime.Now.ToString("yyyy-MM-dd");
                if (strAccountDay == strNowDay)
                    return true;
            }
            return false;
        }

        /// <summary>
        /// 验证网址百度是否收入
        /// </summary>
        /// <param name="strUrl"></param>
        public static bool CheckBaiDu(string strUrl)
        {
            strUrl = string.Format(strBaiduUrl, Common.UrlEncode(strUrl), 1);
            string strHtml = Common.GetPageHTML(strUrl);
            if (!string.IsNullOrEmpty(strHtml))
            {
                //分析页面
                BaseAction baseAction = new BaseAction();
                baseAction.CreateHtmlDoc(strHtml);
                //分析列表
                HtmlElement eleItem = baseAction.htmldoc.GetElementById("content_left");
                if (eleItem != null)
                {
                    HtmlElement firstItem = eleItem.FirstChild;
                    if (firstItem != null && firstItem.GetAttribute("id") == "1")
                    {
                        return true;
                    }
                }
                baseAction.DisDoc();
            }
            return false;
        }
    }
}
